package com.ustadzmenjawab.server;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;

//import org.apache.lucene.analysis.Analyzer;
//import org.apache.lucene.analysis.id.IndonesianAnalyzer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.similar.MoreLikeThis;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory;
import org.apache.lucene.util.Version;

import com.ustadzmenjawab.indexing.UstadzMenjawabSynAnalyzer;
import com.ustadzmenjawab.shared.Article;
import com.ustadzmenjawab.shared.Constant;
import com.ustadzmenjawab.shared.SearchResult;

public class LuceneAccess extends DBAccess {

	static Directory fsdir = null;
	static SearcherManager mgr = null;
	static Thread reopener = null;
	static final long REOPEN_PERIOD = 3 * 1000;
	static{
		try {
			fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			mgr = new SearcherManager(fsdir,	new CustomSearchWarmer(),null); 
			reopener = new Thread(new Runnable(){

				@Override
				public void run() {
					while(true){
						if(mgr != null){
							try {
								mgr.maybeReopen();
								Thread.sleep(REOPEN_PERIOD);
							} catch (IOException e) {
								e.printStackTrace();
							} catch (InterruptedException e) {
								e.printStackTrace();
							}
						}
					}
				}
				
			});
			reopener.start();	
		} catch (IOException e) {
			e.printStackTrace();
			
		} catch (Exception e){
			e.printStackTrace();
		}
		 
	}
	public LuceneAccess(){

	}
	
	public void releaseSearcher(IndexSearcher searcher){
		if(searcher != null){
			try {
				mgr.release(searcher);
			} catch (IOException e) {
				e.printStackTrace();
			}
			searcher = null;
		}
	}
	public float getMaxScoreForOneField(String field,String keyword){
		//Directory fsdir = null;
		IndexSearcher searcher = null;
		try{
			//fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {field,"offline"};			
			String querystr = buildQueryStr(keyword);
			String [] finalQuery = new String[]{querystr,"N"};
			BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST
			};
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35, finalQuery, fields,flags, a);
			//searcher = new IndexSearcher(fsdir, true);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			TopScoreDocCollector collector = TopScoreDocCollector.create(1, true);
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			if(hits != null && hits.length > 0){
			//	System.out.println("Max score "+ field +","+hits[0].score);
				return hits[0].score;
			}
			return 0;
		}catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			releaseSearcher(searcher);
		}
		return 0.0f;
	}
	
	public float getMaxScoreSourceForOneField(String field,String keyword,String source){
		//Directory fsdir = null;
		IndexSearcher searcher = null;
		try{
			//fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {field,"offline","sumber"};			
			String querystr = buildQueryStr(keyword);
			String [] finalQuery = new String[]{querystr,"N",source};
			BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST,
					BooleanClause.Occur.MUST
			};
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35, finalQuery, fields,flags, a);
			//searcher = new IndexSearcher(fsdir, true);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			TopScoreDocCollector collector = TopScoreDocCollector.create(1, true);
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			if(hits != null && hits.length > 0){
	//			System.out.println("Max source score "+ field +","+hits[0].score);
				return hits[0].score;
			}
			return 0;
		}catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			
			releaseSearcher(searcher);

		}
		return 0.0f;
	}
	@Override
	public List<SearchResult> searchDBForKeyword(String keyword, String ipAddr){
		Connection conn = DBConnFactory.getConnection();
		IndexSearcher searcher = null;
		//Directory fsdir = null;
		try {
			insertUserQuery(keyword,ipAddr,conn);
			//fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {"judul","artikel","pertanyaan","offline"};			
			String querystr = buildQueryStr(keyword);
			if(querystr == null || querystr.length() == 0)
				return null;
			String [] finalQuery = new String[]{querystr,querystr,querystr,"N"};
			BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST
			};
			//IndexReader reader = IndexReader.open(fsdir, true);
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
	
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35, finalQuery, fields,flags, a);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			//searcher = new IndexSearcher(reader);
			TopScoreDocCollector collector = TopScoreDocCollector.create(Constant.HIT_PER_PAGE, true);
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs().scoreDocs;

			List<SearchResult> retls = new ArrayList<SearchResult>();
			for(int i=0;i<hits.length;++i) {
	//			System.out.println("new normalized Doc score "+ hits[i].score);
				if(hits[i].score < Constant.MIN_HIT_SCORE)
					continue;
				int docId = hits[i].doc;
				Document d = searcher.doc(docId);
				SearchResult s = new SearchResult();
				s.setId(Integer.parseInt(d.get("id")));
				s.setTitle(d.get("judul"));
				s.setSource(d.get("sumber"));
				String bestFragment = FragmentUtil.getBestFragments(querystr, s.getTitle() + "\n"+ d.get("artikel"));
				s.setSummary(bestFragment);
				s.setUrl(d.get("url"));
				s.setUstadz(d.get("ustadz"));
				s.setOffline(d.get("offline"));
				retls.add(s);
			}
			if(retls.size() == 0)
				return null;
			return retls;
		}  catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			DBConnFactory.closeConnection(conn);
			releaseSearcher(searcher);
		}
		return null;
	}
	@Override
	public List<SearchResult> searchDBForKeywordWithSource(String keyword,String source){
		Connection conn = DBConnFactory.getConnection();
		IndexSearcher searcher = null;
//		Directory fsdir = null;
		try {
			insertUserQuery(keyword +" site: "+source,"",conn);
			//fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {"artikel","pertanyaan","judul","sumber","offline"};			
			String querystr = buildQueryStr(keyword);
			String[] queries = {querystr,querystr,querystr,source,"N"};
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST,
					BooleanClause.Occur.MUST
			};
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35,queries,fields,flags,a);
			mgr.maybeReopen();
			searcher = mgr.acquire();
//			searcher = new IndexSearcher(fsdir, true);
//			searcher.setSimilarity(new NormalizedSimilarity(mapMaxScore));
			TopScoreDocCollector collector = TopScoreDocCollector.create(Constant.HIT_PER_PAGE, true);
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			List<SearchResult> retls = new ArrayList<SearchResult>();
			for(int i=0;i<hits.length;++i) {
				if(hits[i].score < Constant.MIN_HIT_SCORE)
					continue;
				int docId = hits[i].doc;
				Document d = searcher.doc(docId);
				SearchResult s = new SearchResult();
				s.setId(Integer.parseInt(d.get("id")));
				s.setTitle(d.get("judul"));
				s.setSource(d.get("sumber"));
				String bestFragment = FragmentUtil.getBestFragments(querystr, s.getTitle() + "\n"+ d.get("artikel"));
				s.setSummary(bestFragment);
				s.setUrl(d.get("url"));
				s.setUstadz(d.get("ustadz"));
				s.setOffline(d.get("offline"));
				retls.add(s);
			}
			if(retls.size() == 0)
				return null;
			return retls;
		} catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			DBConnFactory.closeConnection(conn);
			releaseSearcher(searcher);

		}
		return null;
	}
	@Override
	public List<SearchResult> searchDBForKeyword(String keyword,int start, int size){
		Connection conn = DBConnFactory.getConnection();
		//Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
			insertUserQuery(keyword +" page: "+start,"",conn);
			//fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {"artikel","pertanyaan","judul","offline"};			
			String querystr = buildQueryStr(keyword);
			String [] finalQuery = new String[]{querystr,querystr,querystr,"N"};
			BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST
			};
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35, finalQuery, fields, flags, a);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			//searcher = new IndexSearcher(fsdir, true);
			TopScoreDocCollector collector = TopScoreDocCollector.create(Constant.MAX_HIT_NUM, true);
			searcher.search(q, collector);
			
			ScoreDoc[] hits = collector.topDocs(start,size).scoreDocs;
			List<SearchResult> retls = new ArrayList<SearchResult>();
			for(int i=0;i<hits.length;++i) {
				if(hits[i].score < Constant.MIN_HIT_SCORE)
					continue;
				int docId = hits[i].doc;
				Document d = searcher.doc(docId);
				SearchResult s = new SearchResult();
				s.setId(Integer.parseInt(d.get("id")));
				s.setTitle(d.get("judul"));
				s.setSource(d.get("sumber"));
				String bestFragment = FragmentUtil.getBestFragments(querystr, s.getTitle() + "\n"+ d.get("artikel"));
				s.setSummary(bestFragment);
				s.setUrl(d.get("url"));
				s.setUstadz(d.get("ustadz"));
				s.setOffline(d.get("offline"));
				retls.add(s);
			}
			if(retls.size() == 0)
				return null;
			return retls;
		}  catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			DBConnFactory.closeConnection(conn);
			releaseSearcher(searcher);

		}
		return null;
	}
	@Override
	public List<String> getSourcesForResult(String keyword){
		Connection conn = DBConnFactory.getConnection();
		IndexSearcher searcher = null;
		//Directory fsdir = null;
		try {
		//	fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {"judul","artikel","pertanyaan","offline"};			
			String querystr = buildQueryStr(keyword);
			String [] finalQuery = new String[]{querystr,querystr,querystr,"N"};
			BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST
			};

			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			MultiFieldQueryParser queryParser = new MultiFieldQueryParser(Version.LUCENE_35,fields,a);
			queryParser.setFuzzyMinSim(Constant.FUZZY_MIN_SIM);
			queryParser.setFuzzyPrefixLength(Constant.FUZZY_PREFIX_MAX);
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35, finalQuery, fields,flags, a);
			//searcher = new IndexSearcher(fsdir, true);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			TopScoreDocCollector collector = TopScoreDocCollector.create(Constant.MAX_HIT_NUM, true);
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			if(collector.topDocs().getMaxScore() < Constant.MIN_HIT_SCORE)
				return null;
			HashSet<String> retls = new HashSet<String>();
			for(int i=0;i<hits.length;++i) {
				if(hits[i].score < Constant.MIN_HIT_SCORE)
					continue;
				int docId = hits[i].doc;
				
				Document d = searcher.doc(docId);
				retls.add(d.get("sumber"));
				if(retls.size() == 5)
					break;
			}
			if(retls.size() == 0)
				return null;
			List<String> ret = new ArrayList<String>();
			for(String sumber : retls){
				ret.add(sumber);
			}
			return ret;
		}  catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			DBConnFactory.closeConnection(conn);
			releaseSearcher(searcher);
		}
		return null;

	}
	@Override
	public List<SearchResult> searchDBForKeyword(String keyword,String source,  int start, int size){
		Connection conn = DBConnFactory.getConnection();
	//	Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
			insertUserQuery(keyword +" site: "+source +" page:"+start,"",conn);
		//	fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {"artikel","pertanyaan","judul","sumber","offline"};			
			String querystr = buildQueryStr(keyword);
			String[] queries = {querystr,querystr,querystr,source,"N"};
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST,
					BooleanClause.Occur.MUST
			};
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35,queries,fields,flags,a);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			//			searcher = new IndexSearcher(fsdir, true);
			TopScoreDocCollector collector = TopScoreDocCollector.create(Constant.MAX_HIT_NUM, true);
			searcher.search(q, collector);
//			searcher.setSimilarity(new NormalizedSimilarity(mapMaxScore));
			ScoreDoc[] hits = collector.topDocs(start,size).scoreDocs;
			List<SearchResult> retls = new ArrayList<SearchResult>();
			for(int i=0;i<hits.length;++i) {
				if(hits[i].score < Constant.MIN_HIT_SCORE)
					continue;
				int docId = hits[i].doc;
				Document d = searcher.doc(docId);
				SearchResult s = new SearchResult();
				s.setId(Integer.parseInt(d.get("id")));
				s.setTitle(d.get("judul"));
				s.setSource(d.get("sumber"));
				String bestFragment = FragmentUtil.getBestFragments(querystr, s.getTitle() + "\n"+ d.get("artikel"));
				s.setSummary(bestFragment);
				s.setUrl(d.get("url"));
				s.setUstadz(d.get("ustadz"));
				s.setOffline(d.get("offline"));
				retls.add(s);
			}
			if(retls.size() == 0)
				return null;
			return retls;
		} catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			DBConnFactory.closeConnection(conn);
			releaseSearcher(searcher);
		}
		return null;
	}
	@Override
	public List<SearchResult> searchSimilarQuestion(String keyword,int qid){
		Connection conn = DBConnFactory.getConnection();
		String querystr = buildQueryStr(keyword);
	//	Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
			insertUserQuery(keyword +" similar: "+qid,"",conn);
	//		fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {"artikel",
					//					"pertanyaan",
					//					"judul",
			"offline"};
			Article artikel = getOneArticle(qid);
			if(artikel == null)
				return null;
			String content = cleanPunctuation(cleanStopWord(artikel.getArtikel()));

			//			String pertanyaan = cleanPunctuation(cleanStopWord(artikel.getPertanyaan()));
			//			String judul = cleanPunctuation(cleanStopWord(artikel.getJudul()));
			BooleanClause.Occur[] flags = {
					BooleanClause.Occur.SHOULD,
					//					BooleanClause.Occur.SHOULD,
					//					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST
			};
			String[] qs = new String[]{content,
					//					pertanyaan,
					//					judul,
			"N"};
			mgr.maybeReopen();
			searcher = mgr.acquire();
		//	searcher = new IndexSearcher(fsdir, true);
			IndexReader reader = searcher.getIndexReader();
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35,qs,fields,flags,a);
			//			MoreLikeThis mlt = new MoreLikeThis(reader);
			//			mlt.setAnalyzer(a);
			//			int docID = getDocumentID(qid);
			//			Query q = mlt.like(docID);
			content = retrieveSignificantTerms(content,reader);
			TopScoreDocCollector collector = TopScoreDocCollector.create(Constant.HIT_PER_PAGE, true);
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			List<SearchResult> retls = new ArrayList<SearchResult>();
			for(int i=0;i<hits.length;++i) {
				if(hits[i].score < Constant.MIN_HIT_SCORE)
					continue;
				int docId = hits[i].doc;
				Document d = searcher.doc(docId);
				SearchResult s = new SearchResult();
				s.setId(Integer.parseInt(d.get("id")));
				s.setTitle(d.get("judul"));
				s.setSource(d.get("sumber"));
				String bestFragment = FragmentUtil.getBestFragments(querystr, s.getTitle() + "\n"+ d.get("artikel"));
				s.setSummary(bestFragment);
				s.setUrl(d.get("url"));
				s.setUstadz(d.get("ustadz"));
				s.setOffline(d.get("offline"));
				retls.add(s);
			}
			if(retls.size() == 0)
				return null;
			return retls;
		} catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} finally{
			DBConnFactory.closeConnection(conn);
			releaseSearcher(searcher);
		}
		return null;
	}

	public List<SearchResult> searchSimilarQuestion(String keyword,int qid, int start, int size){
		Connection conn = DBConnFactory.getConnection();
		String querystr = buildQueryStr(keyword);
	//	Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
			insertUserQuery(keyword +" similar: "+qid +" page: "+start,"",conn);
	//		fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
	//		searcher = new IndexSearcher(fsdir, true);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			//String[] fields  = {"artikel","pertanyaan","judul","offline"};
			String[] fields  = {"artikel","offline"};
			Article artikel = getOneArticle(qid);
			if(artikel == null)
				return null;
			String content = cleanPunctuation(cleanStopWord(artikel.getArtikel()));
			//			String pertanyaan = cleanPunctuation(cleanStopWord(artikel.getPertanyaan()));
			//			String judul = cleanPunctuation(cleanStopWord(artikel.getJudul()));
			content = retrieveSignificantTerms(content,searcher.getIndexReader());
			BooleanClause.Occur[] flags = {
					BooleanClause.Occur.SHOULD,
					//				BooleanClause.Occur.SHOULD,
					//				BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST
			};
			String[] qs = new String[]{content,
					//				pertanyaan,
					//				judul,
			"N"};
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35,qs,fields,flags,a);
			//			MoreLikeThis mlt = new MoreLikeThis(searcher.getIndexReader());
			//			mlt.setAnalyzer(a);
			//			int docID = getDocumentID(qid);
			//			if(docID == 0)
			//				return null;
			//			Query q = mlt.like(docID);
			TopScoreDocCollector collector = TopScoreDocCollector.create(Constant.MAX_HIT_NUM, true);
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs(start,size).scoreDocs;
			List<SearchResult> retls = new ArrayList<SearchResult>();
			for(int i=0;i<hits.length;++i) {
				if(hits[i].score < Constant.MIN_HIT_SCORE)
					continue;
				int docId = hits[i].doc;
				Document d = searcher.doc(docId);
				SearchResult s = new SearchResult();
				s.setId(Integer.parseInt(d.get("id")));
				s.setTitle(d.get("judul"));
				s.setSource(d.get("sumber"));
				String bestFragment = FragmentUtil.getBestFragments(querystr, s.getTitle() + "\n"+ d.get("artikel"));
				s.setSummary(bestFragment);
				s.setUrl(d.get("url"));
				s.setUstadz(d.get("ustadz"));
				s.setOffline(d.get("offline"));
				retls.add(s);
			}
			if(retls.size() == 0)
				return null;
			return retls;
		} catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		} finally{
			DBConnFactory.closeConnection(conn);
			releaseSearcher(searcher);
		}
		return null;
	}

	public int getTotalCount(String keyword){
		//Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
		//	fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {"artikel","pertanyaan","judul","offline"};			
			String querystr = buildQueryStr(keyword);
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			String [] finalQuery = new String[]{querystr,querystr,querystr,"N"};
			BooleanClause.Occur[] flags = {
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST
			};

			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35, finalQuery, fields,flags, a);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			//searcher = new IndexSearcher(fsdir, true);			
			TopDocs topDocs = searcher.search(q, Constant.MAX_HIT_NUM);
			if(topDocs.getMaxScore() < Constant.MIN_HIT_SCORE)
				return 0;
			int countRet = 0;
			ScoreDoc[] hits = topDocs.scoreDocs;
			for(ScoreDoc hit : hits){
				if(hit.score > Constant.MIN_HIT_SCORE)
					countRet++;
			}
			
			return countRet;
		}catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			releaseSearcher(searcher);
		}
		return 0;
	}

	public int getTotalCount(String keyword, String source){
	//	Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
			//fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			String[] fields  = {"artikel","pertanyaan","judul","sumber","offline"};			
			String querystr = buildQueryStr(keyword);
			String[] queries = {querystr,querystr,querystr,source,"N"};
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			BooleanClause.Occur[] flags = {
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST,
					BooleanClause.Occur.MUST
			};
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35,queries,fields,flags,a);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			//searcher = new IndexSearcher(fsdir, true);
			TopDocs topDocs = searcher.search(q, Constant.MAX_HIT_NUM);
			if(topDocs.getMaxScore() < Constant.MIN_HIT_SCORE)
				return 0;
			int countRet = 0;
			ScoreDoc[] hits = topDocs.scoreDocs;
			for(ScoreDoc hit : hits){
				if(hit.score > Constant.MIN_HIT_SCORE)
					countRet++;
			}
			
			return countRet;
		}catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			e.printStackTrace();
		}finally{
			releaseSearcher(searcher);
		}
		return 0;
	}

	public int getCountSimilarResult(int qid){
	//	Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
	//		fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			mgr.maybeReopen();
			searcher = mgr.acquire();
	//		searcher = new IndexSearcher(fsdir, true);
			Analyzer a = new UstadzMenjawabSynAnalyzer(Version.LUCENE_35);
			String[] fields  = {"artikel",
					//					"pertanyaan",
					//					"judul",
			"offline"};
			Article artikel = getOneArticle(qid);
			if(artikel == null)
				return 0;
			String content = cleanPunctuation(cleanStopWord(artikel.getArtikel()));
			//			String pertanyaan = cleanPunctuation(cleanStopWord(artikel.getPertanyaan()));
			//			String judul = cleanPunctuation(cleanStopWord(artikel.getJudul()));
			BooleanClause.Occur[] flags = {
					BooleanClause.Occur.SHOULD,
					//					BooleanClause.Occur.SHOULD,
					//					BooleanClause.Occur.SHOULD,
					BooleanClause.Occur.MUST
			};
			content = retrieveSignificantTerms(content,searcher.getIndexReader());
			String[] qs = new String[]{content,
					//					pertanyaan,
					//					judul,
			"N"};
			//			MoreLikeThis mlt = new MoreLikeThis(searcher.getIndexReader());
			//			mlt.setAnalyzer(a);
			//			Query q = mlt.like(qid);
			Query q = MultiFieldQueryParser.parse(Version.LUCENE_35,qs,fields,flags,a);
			TopDocs topDocs =searcher.search(q, Constant.MAX_HIT_NUM);
			if(topDocs.getMaxScore() < Constant.MIN_HIT_SCORE)
				return 0;
			int countRet = 0;
			ScoreDoc[] hits = topDocs.scoreDocs;
			for(ScoreDoc hit : hits){
				if(hit.score >= Constant.MIN_HIT_SCORE)
					countRet++;
			}
			
			return countRet;
		}catch (IOException e) {
			e.printStackTrace();
		} catch (ParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally{
			releaseSearcher(searcher);
		}
		return 0;
	}
	@Override
	public Article getOneArticle(int articleID){
		//Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
			QueryParser parser = new QueryParser(Version.LUCENE_35,"id",new UstadzMenjawabSynAnalyzer(Version.LUCENE_35));
			Query q = parser.parse("id:"+articleID);
			//fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			//searcher = new IndexSearcher(fsdir, true);
			TopScoreDocCollector collector = TopScoreDocCollector.create(1, true);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			Article ret = new Article();
			if(hits == null || hits.length == 0)
				return null;
			ScoreDoc sd = hits[0];
			int docId = sd.doc;
			Document d = searcher.doc(docId);
			ret.setId(Integer.parseInt(d.get("id"))) ;
			ret.setJudul(d.get("judul"));
			ret.setPertanyaan(d.get("pertanyaan"));
			ret.setArtikel(d.get("artikel"));
			ret.setKategori(d.get("kategori"));
			ret.setUrl(d.get("ustadz"));
			ret.setSumber(d.get("sumber"));
			ret.setTahun(d.get("tahun"));

			return ret;
		}catch (Exception e) {
			e.printStackTrace();
		}finally{
			releaseSearcher(searcher);
		}
		return null;
	}


	public int getDocumentID(int articleID){
	//	Directory fsdir = null;
		IndexSearcher searcher = null;
		try {
			QueryParser parser = new QueryParser(Version.LUCENE_35,"id",new UstadzMenjawabSynAnalyzer(Version.LUCENE_35));
			Query q = parser.parse("id:"+articleID);
			mgr.maybeReopen();
			searcher = mgr.acquire();
			//fsdir = new NIOFSDirectory(new File(Constant.INDEX_LOC));
			//	searcher = new IndexSearcher(fsdir, true);
			TopScoreDocCollector collector = TopScoreDocCollector.create(1, true);
			searcher.search(q, collector);
			ScoreDoc[] hits = collector.topDocs().scoreDocs;
			
			if(hits == null || hits.length == 0)
				return 0;
			ScoreDoc sd = hits[0];
			int docId = sd.doc;
			return docId;
		}catch (Exception e) {
			e.printStackTrace();
		}finally{
			releaseSearcher(searcher);
		}
		return 0;
	}

	public String buildQueryStr(String keyword){
		if(keyword == null)
			return null;
		Connection conn = DBConnFactory.getConnection();
		String keyword_artikel = cleanStopWord(keyword);
		if(keyword_artikel == null)
			return null;
		keyword_artikel = cleanPunctuation(keyword_artikel);
		keyword_artikel = expandQuery(keyword_artikel, conn);
		try {
			String querystr = keyword_artikel;
			querystr = querystr.trim();
			return querystr;
		}finally{
			DBConnFactory.closeConnection(conn);
		}
	}

	public String retrieveSignificantTerms(String keyword, IndexReader ir) throws IOException{
		if(keyword == null || keyword.length() == 0)
			return null;
		String[] origs = keyword.split(" ");
		int numTerms = keyword.split(" ").length;
		if(numTerms > Constant.MAX_SIGNIFICANT_TERMS){
			MoreLikeThis mlt = new MoreLikeThis(ir);
			Reader reader = new StringReader(keyword);
			String[] signs;
			signs = mlt.retrieveInterestingTerms(reader);
			if(signs == null || signs.length == 0){
				signs = Arrays.copyOf(origs, Constant.MAX_SIGNIFICANT_TERMS);
			}
			String ret = implode(signs," ");
			System.out.println("Significant Terms:" + ret);
			reader.close();

			return ret;
		}
		return keyword;
	}

	public static String implode(String[] ary, String delim) {
		String out = "";
		for(int i=0; i<ary.length; i++) {
			if(i!=0) { out += delim; }
			out += ary[i];
		}
		return out;
	}
}
